#coding=utf-8
from bs4 import BeautifulSoup
import os
import ConfigParser
import requests
import re
from brokerRedis import brokerRedis
from masterRedis import masterRedis
from record import record
import sys
class application:
    def __init__(self):
        self.config=ConfigParser.ConfigParser()
        self.brokerRedis=brokerRedis()
        self.masterRedis=masterRedis()
        path = os.path.split(os.path.realpath(__file__))[0] + '/config.conf'
        self.config.read(path)
        self.recordMong=record()
    
    def urlByRedis(self):
        url=self.masterRedis.getUrl()
        self.getLinks(url[1])
    
    def getLinks(self,url):
        res=requests.get(url)
        if self.config.get("encoding","encoding")!="":
            res.encoding="utf-8"
        soup=BeautifulSoup(res.text,"lxml")
        links=soup.find_all("a",attrs={"href":re.compile(self.config.get("urlregular","ahref"))})
        for link in links:
            self.brokerRedis.saveUrl(link.get("href"))
        self.getContent(res,url)
        
    
    def getContent(self,res,url):
        json={}
        soup=BeautifulSoup(res.text)
        for result in self.config.items("htmlCss"):
            className=self.config.get("htmlCss",result[0])
            results=soup.select(className)
            term=self.config.get("condition",className)
            if results:
                if (term==None or term==""):
                        content=[value.get_text() for value in results]
                        json[result[0]]=content
                else:
                    try:
                        strArr=term.split(",")
                        reValue=[]
                        for value in range(1,len(strArr)):
                            reValue.append(results[int(strArr[0])].contents[int(value)].get_text())
                        json[result[0]]=reValue
                    except Exception:
                        print "error str len"
                        
            else:
                print "no match when url="+url
        if json:
            self.recordMong.add(json)
        nextUrl=self.masterRedis.getUrl()[1]
        self.getLinks(nextUrl)
        
if __name__ == "__main__" :
    sys.setrecursionlimit(100000)
    application=application()
    application.urlByRedis()